{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"SG-QAFFN_cross_entropy.ipynb","version":"0.3.2","provenance":[{"file_id":"1XIn68D4-toNoNqJ_Zk7GJIrJ2AMXL-sX","timestamp":1556127036366}],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"fBYR7rLZBYNS","colab_type":"code","outputId":"415d3624-0bb8-4052-f041-923a9615b213","executionInfo":{"status":"ok","timestamp":1556311483423,"user_tz":420,"elapsed":19946,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}},"colab":{"base_uri":"https://localhost:8080/","height":121}},"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/gdrive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/gdrive\n"],"name":"stdout"}]},{"metadata":{"id":"ueDiJQmTB17f","colab_type":"code","outputId":"bd11ec09-f633-4df7-a691-bb984c732879","executionInfo":{"status":"ok","timestamp":1556311533186,"user_tz":420,"elapsed":44012,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}},"colab":{"base_uri":"https://localhost:8080/","height":541}},"cell_type":"code","source":["# install tf 2.0\n","from __future__ import absolute_import, division, print_function, unicode_literals\n","\n","!pip install tensorflow-gpu==2.0.0-alpha0\n","import tensorflow as tf\n","# tf.compat.v1.disable_eager_execution()\n","\n","print(tf.__version__)"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Collecting tensorflow-gpu==2.0.0-alpha0\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/66/32cffad095253219d53f6b6c2a436637bbe45ac4e7be0244557210dc3918/tensorflow_gpu-2.0.0a0-cp36-cp36m-manylinux1_x86_64.whl (332.1MB)\n","\u001b[K    100% |████████████████████████████████| 332.1MB 34kB/s \n","\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.9)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)\n","Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)\n","Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.1)\n","Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)\n","Collecting google-pasta>=0.1.2 (from tensorflow-gpu==2.0.0-alpha0)\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/bb/f1bbc131d6294baa6085a222d29abadd012696b73dcbf8cf1bf56b9f082a/google_pasta-0.1.5-py3-none-any.whl (51kB)\n","\u001b[K    100% |████████████████████████████████| 61kB 29.4MB/s \n","\u001b[?25hRequirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.7)\n","Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)\n","Collecting tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 (from tensorflow-gpu==2.0.0-alpha0)\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/13/82/f16063b4eed210dc2ab057930ac1da4fbe1e91b7b051a6c8370b401e6ae7/tf_estimator_nightly-1.14.0.dev2019030115-py2.py3-none-any.whl (411kB)\n","\u001b[K    100% |████████████████████████████████| 419kB 13.4MB/s \n","\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.16.3)\n","Collecting tb-nightly<1.14.0a20190302,>=1.14.0a20190301 (from tensorflow-gpu==2.0.0-alpha0)\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a9/51/aa1d756644bf4624c03844115e4ac4058eff77acd786b26315f051a4b195/tb_nightly-1.14.0a20190301-py3-none-any.whl (3.0MB)\n","\u001b[K    100% |████████████████████████████████| 3.0MB 10.2MB/s \n","\u001b[?25hRequirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n","Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.12.0)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (40.9.0)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.2)\n","Installing collected packages: google-pasta, tf-estimator-nightly, tb-nightly, tensorflow-gpu\n","Successfully installed google-pasta-0.1.5 tb-nightly-1.14.0a20190301 tensorflow-gpu-2.0.0a0 tf-estimator-nightly-1.14.0.dev2019030115\n","2.0.0-alpha0\n"],"name":"stdout"}]},{"metadata":{"id":"uDqguCPeB39D","colab_type":"code","colab":{}},"cell_type":"code","source":["import os\n","from glob import glob\n","\n","import numpy as np\n","import pandas as pd\n","import tensorflow as tf\n","from sklearn.model_selection import train_test_split\n","\n","SEED = 42\n","\n","\n","def _float_list_feature(value):\n","    \"\"\"Returns a float_list from a float / double.\"\"\"\n","    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n","\n","\n","def _int64_list_feature(value):\n","    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n","    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n","\n","\n","def _int64_feature(value):\n","    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n","    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n","\n","\n","def create_generator_for_ffn(\n","        file_list,\n","        mode='train'):\n","\n","    # file_list = glob(os.path.join(data_dir, '*.csv'))\n","\n","    for full_file_path in file_list:\n","        # full_file_path = os.path.join(data_dir, file_name)\n","        if not os.path.exists(full_file_path):\n","            raise FileNotFoundError(\"File %s not found\" % full_file_path)\n","        df = pd.read_csv(full_file_path, encoding='utf8')\n","\n","        # so train test split\n","        if mode == 'train':\n","            df, _ = train_test_split(df, test_size=0.2, random_state=SEED)\n","        else:\n","            _, df = train_test_split(df, test_size=0.2, random_state=SEED)\n","\n","        for _, row in df.iterrows():\n","            q_vectors = np.fromstring(row.question_bert.replace(\n","                '[[', '').replace(']]', ''), sep=' ')\n","            a_vectors = np.fromstring(row.answer_bert.replace(\n","                '[[', '').replace(']]', ''), sep=' ')\n","            vectors = np.stack([q_vectors, a_vectors], axis=0)\n","            if mode in ['train', 'eval']:\n","                yield vectors, 1\n","            else:\n","                yield vectors\n","\n","\n","def ffn_serialize_fn(features):\n","    features_tuple = {'features': _float_list_feature(\n","        features[0].flatten()), 'labels': _int64_feature(features[1])}\n","    example_proto = tf.train.Example(\n","        features=tf.train.Features(feature=features_tuple))\n","    return example_proto.SerializeToString()\n","\n","\n","def make_tfrecord(data_dir, generator_fn, serialize_fn, suffix='', **kwargs):\n","    \"\"\"Function to make TF Records from csv files\n","    This function will take all csv files in data_dir, convert them\n","    to tf example and write to *_{suffix}_train/eval.tfrecord to data_dir.\n","\n","    Arguments:\n","        data_dir {str} -- dir that has csv files and store tf record\n","        generator_fn {fn} -- A function that takes a list of filepath and yield the\n","        parsed recored from file.\n","        serialize_fn {fn} -- A function that takes output of generator fn and convert to tf example\n","\n","    Keyword Arguments:\n","        suffix {str} -- suffix to add to tf record files (default: {''})\n","    \"\"\"\n","    file_list = glob(os.path.join(data_dir, '*.csv'))\n","    train_tf_record_file_list = [\n","        f.replace('.csv', '_{0}_train.tfrecord'.format(suffix)) for f in file_list]\n","    test_tf_record_file_list = [\n","        f.replace('.csv', '_{0}_eval.tfrecord'.format(suffix)) for f in file_list]\n","    for full_file_path, train_tf_record_file_path, test_tf_record_file_path in zip(file_list, train_tf_record_file_list, test_tf_record_file_list):\n","        print('Converting file {0} to TF Record'.format(full_file_path))\n","        with tf.io.TFRecordWriter(train_tf_record_file_path) as writer:\n","            for features in generator_fn([full_file_path], mode='train', **kwargs):\n","                example = serialize_fn(features)\n","                writer.write(example)\n","        with tf.io.TFRecordWriter(test_tf_record_file_path) as writer:\n","            for features in generator_fn([full_file_path], mode='eval', **kwargs):\n","                example = serialize_fn(features)\n","                writer.write(example)\n","\n","\n","def create_dataset_for_ffn(\n","        data_dir,\n","        mode='train',\n","        hidden_size=768,\n","        shuffle_buffer=10000,\n","        prefetch=10000,\n","        batch_size=32):\n","\n","    tfrecord_file_list = glob(os.path.join(\n","        data_dir, '*_FFN_{0}.tfrecord'.format((mode))))\n","    if not tfrecord_file_list:\n","        print('TF Record not found')\n","        make_tfrecord(\n","            data_dir, create_generator_for_ffn,\n","            ffn_serialize_fn, 'FFN')\n","\n","    dataset = tf.data.TFRecordDataset(tfrecord_file_list)\n","\n","    def _parse_ffn_example(example_proto):\n","        feature_description = {\n","            'features': tf.io.FixedLenFeature([2*768], tf.float32),\n","            'labels': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n","        }\n","        feature_dict = tf.io.parse_single_example(\n","            example_proto, feature_description)\n","        return tf.reshape(feature_dict['features'], (2, 768)), feature_dict['labels']\n","    dataset = dataset.map(_parse_ffn_example)\n","\n","    dataset = dataset.shuffle(shuffle_buffer)\n","\n","    dataset = dataset.prefetch(prefetch)\n","\n","    dataset = dataset.batch(batch_size)\n","    return dataset"],"execution_count":0,"outputs":[]},{"metadata":{"id":"fFByODECB4lV","colab_type":"code","colab":{}},"cell_type":"code","source":["from __future__ import absolute_import, division, print_function, unicode_literals\n","\n","import os\n","import pandas as pd\n","from sklearn.model_selection import train_test_split\n","import numpy as np\n","\n","import tensorflow as tf\n","import tensorflow.keras.backend as K\n","\n","\n","class FFN(tf.keras.layers.Layer):\n","    def __init__(\n","            self,\n","            hidden_size=768,                                                                #SG edit from 768 4-24-19\n","            dropout=0.2,\n","            residual=True,\n","            name='FFN',\n","            **kwargs):\n","        \"\"\"Simple Dense wrapped with various layers\n","        \"\"\"\n","\n","        super(FFN, self).__init__(name=name, **kwargs)\n","        self.hidden_size = hidden_size\n","        self.dropout = dropout\n","        self.residual = residual\n","        self.ffn_layer = tf.keras.layers.Dense(\n","            units=hidden_size,\n","            use_bias=True\n","        )\n","\n","    def call(self, inputs):\n","        ffn_embedding = self.ffn_layer(inputs)\n","        ffn_embedding = tf.keras.layers.ReLU()(ffn_embedding)\n","        if self.dropout > 0:\n","            ffn_embedding = tf.keras.layers.Dropout(\n","                self.dropout)(ffn_embedding)\n","#         ffn_embedding = self.ffn_layer(inputs)  #SG edit from 768 4-24-19\n","#         ffn_embedding = tf.keras.layers.ReLU()(ffn_embedding)  #SG edit from 768 4-24-19\n","#         if self.dropout > 0:  #SG edit from 768 4-24-19\n","#             ffn_embedding = tf.keras.layers.Dropout(  #SG edit from 768 4-24-19\n","#                 self.dropout)(ffn_embedding)  #SG edit from 768 4-24-19\n","\n","\n","        if self.residual:\n","            ffn_embedding += inputs\n","        return ffn_embedding\n","\n","\n","class MedicalQAModel(tf.keras.Model):\n","    def __init__(self, name=''):\n","        super(MedicalQAModel, self).__init__(name=name)\n","        self.q_ffn = FFN(name='QFFN', input_shape=(768,))\n","        self.a_ffn = FFN(name='AFFN', input_shape=(768,))\n","\n","    def call(self, inputs):\n","        q_bert_embedding, a_bert_embedding = tf.unstack(inputs, axis=1)\n","        q_embedding, a_embedding = self.q_ffn(\n","            q_bert_embedding), self.a_ffn(a_bert_embedding)\n","        return tf.stack([q_embedding, a_embedding], axis=1)\n","\n","\n","class BioBert(tf.keras.Model):\n","    def __init__(self, name=''):\n","        super(BioBert, self).__init__(name=name)\n","\n","    def call(self, inputs):\n","\n","        # inputs is dict with input features\n","        input_ids, input_masks, segment_ids = inputs\n","        # pass to bert\n","        # with shape of (batch_size/2*batch_size, max_seq_len, hidden_size)\n","        # TODO(Alex): Add true bert model\n","        # Input: input_ids, input_masks, segment_ids all with shape (None, max_seq_len)\n","        # Output: a tensor with shape (None, max_seq_len, hidden_size)\n","        fake_bert_output = tf.expand_dims(tf.ones_like(\n","            input_ids, dtype=tf.float32), axis=-1)*tf.ones([1, 1, 768], dtype=tf.float32)\n","        max_seq_length = tf.shape(fake_bert_output)[-2]\n","        hidden_size = tf.shape(fake_bert_output)[-1]\n","\n","        bert_output = tf.reshape(\n","            fake_bert_output, (-1, 2, max_seq_length, hidden_size))\n","        return bert_output\n","\n","\n","class MedicalQAModelwithBert(tf.keras.Model):\n","    def __init__(\n","            self,\n","            hidden_size=768,\n","            dropout=0.2,\n","            residual=True,\n","            activation=tf.keras.layers.ReLU(),\n","            name=''):\n","        super(MedicalQAModelwithBert, self).__init__(name=name)\n","        self.biobert = BioBert()\n","        self.q_ffn_layer = FFN(\n","            hidden_size=hidden_size,\n","            dropout=dropout,\n","            residual=residual,\n","            activation=activation)\n","        self.a_ffn_layer = FFN(\n","            hidden_size=hidden_size,\n","            dropout=dropout,\n","            residual=residual,\n","            activation=activation)\n","\n","    def _avg_across_token(self, tensor):\n","        if tensor is not None:\n","            tensor = tf.reduce_mean(tensor, axis=1)\n","        return tensor\n","\n","    def call(self, inputs):\n","\n","        q_bert_embedding, a_bert_embedding = self.biobert(inputs)\n","\n","        # according to USE, the DAN network average embedding across tokens\n","        q_bert_embedding = self._avg_across_token(q_bert_embedding)\n","        a_bert_embedding = self._avg_across_token(a_bert_embedding)\n","\n","        q_embedding = self.q_ffn_layer(q_bert_embedding)\n","        a_embedding = self.a_ffn_layer(a_bert_embedding)\n","\n","        return tf.stack([q_embedding, a_embedding], axis=1)\n","\n","      \n","      \n","# def qa_pair_cross_entropy_loss(y_true, y_pred):\n","#     y_true = tf.eye(tf.shape(y_pred)[0])\n","#     q_embedding, a_embedding = tf.unstack(y_pred, axis=1)\n","#     similarity_matrix = tf.matmul(\n","#         q_embedding, a_embedding, transpose_b=True)\n","#     similarity_matrix_logits = tf.math.sigmoid(similarity_matrix)\n","#     return tf.keras.losses.categorical_crossentropy(y_true, similarity_matrix_logits, from_logits=True)\n","\n","def qa_pair_cross_entropy_loss(y_true, y_pred):\n","    y_true = tf.eye(tf.shape(y_pred)[0])\n","    q_embedding, a_embedding = tf.unstack(y_pred, axis=1)\n","    similarity_matrix = tf.matmul(\n","        a = q_embedding, b = a_embedding, transpose_b=True)\n","    similarity_matrix_softmaxed = tf.nn.softmax(similarity_matrix)\n","    K.print_tensor(similarity_matrix_softmaxed, message=\"similarity_matrix_softmaxed is: \")\n","    return tf.keras.losses.categorical_crossentropy(y_true, similarity_matrix_softmaxed, from_logits=False)\n","\n","#     y_true = tf.reshape(tf.eye(tf.shape(y_pred)[0])*2-1, (-1,))\n","#     q_embedding, a_embedding = tf.unstack(y_pred, axis=1)\n","#     similarity_matrix = tf.nn.softmax(tf.matmul(\n","#         q_embedding, a_embedding, transpose_b=True))\n","#     similarity_vector = tf.reshape(similarity_matrix, (-1, 1))\n","#     return tf.nn.softmax_cross_entropy_with_logits(similarity_vector, y_true)\n","\n","#to try, with and without softmax\n","# catagorical cross entropy vs binary cross entropy\n","#with and without sigmoid pre transformation\n","#1 layer vs 2 layer \n","\n","#prioritize what he said. so softmax, then catagorical vs binary \n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Op9nhWzEB-2V","colab_type":"code","colab":{}},"cell_type":"code","source":["# training config\n","batch_size = 128\n","num_epochs=35\n","learning_rate=0.0001\n","validation_split=0.2\n","shuffle_buffer=50000\n","prefetch=50000\n","data_path='/content/gdrive/My Drive/mqa_tf_record'\n","model_path = '/content/gdrive/My Drive/mqa_models/ffn_model_cross_entropy'"],"execution_count":0,"outputs":[]},{"metadata":{"id":"b7QrPxwVB_U2","colab_type":"code","colab":{}},"cell_type":"code","source":["  d = create_dataset_for_ffn(\n","      data_path, batch_size=batch_size, shuffle_buffer=shuffle_buffer, prefetch=prefetch)\n","  eval_d = create_dataset_for_ffn(\n","      data_path, batch_size=batch_size, mode='eval')\n","  medical_qa_model = MedicalQAModel()\n","  optimizer = tf.keras.optimizers.Adam(lr=learning_rate)\n","  medical_qa_model.compile(\n","      optimizer=optimizer, loss=qa_pair_cross_entropy_loss)\n","\n","  epochs = num_epochs\n","  loss_metric = tf.keras.metrics.Mean()\n","\n","#   history = medical_qa_model.fit(d, epochs=epochs, validation_data=eval_d)\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Nl6zuquoJQJ1","colab_type":"code","colab":{}},"cell_type":"code","source":["\n","model_path2 = '/content/gdrive/My Drive/mqa_models/ffn_model_cross_entropy.ckpt'\n","\n","checkpoint = tf.keras.callbacks.ModelCheckpoint(model_path2, monitor='loss', verbose=1, save_best_only=True)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"PkFe-Nj5m4JU","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"outputId":"5ae69c24-3b33-4bb9-a08a-5008cf7cfdec","executionInfo":{"status":"ok","timestamp":1556311535315,"user_tz":420,"elapsed":23949,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}}},"cell_type":"code","source":["checkpoint_dir = os.path.dirname(model_path2)\n","print(checkpoint_dir)"],"execution_count":8,"outputs":[{"output_type":"stream","text":["/content/gdrive/My Drive/mqa_models\n"],"name":"stdout"}]},{"metadata":{"id":"uJIvZhXinF4l","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"outputId":"de3f56d8-460b-4660-9f1e-5b5950e0d51e","executionInfo":{"status":"ok","timestamp":1556311536621,"user_tz":420,"elapsed":23153,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}}},"cell_type":"code","source":["medical_qa_model.load_weights(model_path2)"],"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f79ae5e6c18>"]},"metadata":{"tags":[]},"execution_count":9}]},{"metadata":{"id":"zQLRChm_n2t-","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":662},"outputId":"85fb6c6f-9bd7-4291-983b-58c750705b65","executionInfo":{"status":"error","timestamp":1556311550052,"user_tz":420,"elapsed":12072,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}}},"cell_type":"code","source":["history = medical_qa_model.fit(d, epochs=80, validation_data=eval_d,  callbacks=[checkpoint])"],"execution_count":10,"outputs":[{"output_type":"stream","text":["Epoch 1/80\n"],"name":"stdout"},{"output_type":"stream","text":["WARNING: Logging before flag parsing goes to stderr.\n","W0426 20:45:43.031697 140162562127744 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py:2924: Print (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2018-08-20.\n","Instructions for updating:\n","Use tf.print instead of tf.Print. Note that tf.print returns a no-output operator that directly prints the output. Outside of defuns or eager mode, this operator will not be executed unless it is directly specified in session.run or used as a control dependency for other operators. This is only a concern in graph mode. Below is an example of how to ensure tf.print executes in graph mode:\n","```python\n","    sess = tf.Session()\n","    with sess.as_default():\n","        tensor = tf.range(10)\n","        print_op = tf.print(tensor)\n","        with tf.control_dependencies([print_op]):\n","          out = tf.add(tensor, tensor)\n","        sess.run(out)\n","    ```\n","Additionally, to use tf.print in python 2.7, users must make sure to import\n","the following:\n","\n","  `from __future__ import print_function`\n","\n"],"name":"stderr"},{"output_type":"stream","text":["    179/Unknown - 11s 64ms/step - loss: 1.8521"],"name":"stdout"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-10-d7f789654f0c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmedical_qa_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m80\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_d\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    789\u001b[0m           \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    790\u001b[0m           \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 791\u001b[0;31m           initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m    792\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    793\u001b[0m     \u001b[0;31m# Case 3: Symbolic tensors or Numpy array-like.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1513\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1514\u001b[0m         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1515\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m   1516\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1517\u001b[0m   def evaluate_generator(self,\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    251\u001b[0m       \u001b[0;31m# Callbacks batch begin.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m       \u001b[0mbatch_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'batch'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'size'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 253\u001b[0;31m       \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'begin'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    254\u001b[0m       \u001b[0mprogbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m    228\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delta_ts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt_before_callbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m     \u001b[0mdelta_t_median\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmedian\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delta_ts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    231\u001b[0m     if (self._delta_t_batch > 0. and\n\u001b[1;32m    232\u001b[0m         delta_t_median > 0.95 * self._delta_t_batch and delta_t_median > 0.1):\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36mmedian\u001b[0;34m(a, axis, out, overwrite_input, keepdims)\u001b[0m\n\u001b[1;32m   3495\u001b[0m     \"\"\"\n\u001b[1;32m   3496\u001b[0m     r, k = _ureduce(a, func=_median, axis=axis, out=out,\n\u001b[0;32m-> 3497\u001b[0;31m                     overwrite_input=overwrite_input)\n\u001b[0m\u001b[1;32m   3498\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3499\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36m_ureduce\u001b[0;34m(a, func, **kwargs)\u001b[0m\n\u001b[1;32m   3403\u001b[0m         \u001b[0mkeepdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3404\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3405\u001b[0;31m     \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3406\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3407\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36m_median\u001b[0;34m(a, axis, out, overwrite_input)\u001b[0m\n\u001b[1;32m   3549\u001b[0m         \u001b[0;31m# warn and return nans like mean would\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3550\u001b[0m         \u001b[0mrout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpart\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3551\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_median_nancheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3552\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3553\u001b[0m         \u001b[0;31m# if there are no nans\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/lib/utils.py\u001b[0m in \u001b[0;36m_median_nancheck\u001b[0;34m(data, result, axis, out)\u001b[0m\n\u001b[1;32m   1139\u001b[0m     \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnan\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m...\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1140\u001b[0m     \u001b[0;31m# masked NaN values are ok\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1141\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misMaskedArray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1142\u001b[0m         \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1143\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/ma/core.py\u001b[0m in \u001b[0;36misMaskedArray\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m   6246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6247\u001b[0m     \"\"\"\n\u001b[0;32m-> 6248\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMaskedArray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   6249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{"id":"LVFOo7PJGZNh","colab_type":"code","outputId":"c43614b8-80ff-4334-dc7b-81cfd22f4863","executionInfo":{"status":"ok","timestamp":1556162257404,"user_tz":420,"elapsed":846,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}},"colab":{"base_uri":"https://localhost:8080/","height":574}},"cell_type":"code","source":["import matplotlib.pyplot as plt\n","# summarize history for accuracy\n","# plt.plot(history.history['acc'])\n","# plt.plot(history.history['val_acc'])\n","plt.title('model accuracy')\n","plt.ylabel('accuracy')\n","plt.xlabel('epoch')\n","plt.legend(['train', 'test'], loc='upper left')\n","plt.show()\n","# summarize history for loss\n","plt.plot(history.history['loss'])\n","plt.plot(history.history['val_loss'])\n","plt.title('model loss')\n","plt.ylabel('loss')\n","plt.xlabel('epoch')\n","plt.legend(['train', 'test'], loc='upper left')\n","plt.show()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFnpJREFUeJzt3X20XXV95/H3hxCMSgSbxKkSFNSg\nptYK3qLWtmKhsxBL0NpRsGixDrQqPrTq1FZHWXQe6ljtjJYWorUiIg8ySjOKMkBRxweQIIiCoikV\nufhADA8iGh7kO3/sneZwudnZuWTfe3Lzfq111zp779/e53t/697zOXv/zv6dVBWSJG3JLnNdgCRp\nvBkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFdipJPpjkv/Rs+50khwxdkzTuDApJUieDQtoBJdl1\nrmvQzsOg0NhpL/m8KclVSe5I8g9J/l2STyW5PcmFSR4+0n5VkquT3JrkM0meNLJt/yRfafc7C1g0\n5bl+J8mV7b5fTPKUnjU+L8kVSX6c5IYkJ0zZ/uvt8W5ttx/Trn9wkncluT7JbUk+3647KMnkNP1w\nSPv4hCTnJPlwkh8DxyQ5MMmX2uf4fpK/TbLbyP6/lOSCJDcn+WGSv0jyi0l+mmTJSLsDkqxPsrDP\n766dj0GhcfVC4LeB/YDDgU8BfwEso/m7fS1Akv2AM4DXt9vOA/5Pkt3aF81zgdOAXwA+2h6Xdt/9\ngQ8AfwQsAU4B1iR5UI/67gBeBuwJPA94ZZLnt8d9TFvve9uangpc2e7318DTgF9ra/pPwL09++QI\n4Jz2OU8Hfg78CbAUeCZwMPCqtobFwIXAp4FHAY8HLqqqHwCfAV40ctyXAmdW1d0969BOxqDQuHpv\nVf2wqm4E/h9waVVdUVUbgY8D+7ftXgx8sqouaF/o/hp4MM0L8TOAhcD/rKq7q+oc4LKR5zgOOKWq\nLq2qn1fVqcCd7X6dquozVfW1qrq3qq6iCatnt5tfAlxYVWe0z7uhqq5Msgvwh8DrqurG9jm/WFV3\n9uyTL1XVue1z/qyqLq+qS6rqnqr6Dk3Qbarhd4AfVNW7qmpjVd1eVZe2204FjgZIsgA4iiZMpWkZ\nFBpXPxx5/LNplndvHz8KuH7Thqq6F7gB2KvddmPdd+bL60cePwZ4Q3vp5tYktwJ7t/t1SvL0JBe3\nl2xuA/6Y5p097TH+ZZrdltJc+ppuWx83TKlhvySfSPKD9nLUf+tRA8A/ASuT7Etz1nZbVX15hjVp\nJ2BQaEf3PZoXfACShOZF8kbg+8Be7bpNHj3y+Abgv1bVniM/D6mqM3o870eANcDeVbUHcDKw6Xlu\nAB43zT4/AjZuYdsdwENGfo8FNJetRk2d6vnvgW8CK6rqYTSX5kZreOx0hbdnZWfTnFW8FM8mtBUG\nhXZ0ZwPPS3JwOxj7BprLR18EvgTcA7w2ycIkvwscOLLv+4A/bs8OkuSh7SD14h7Puxi4uao2JjmQ\n5nLTJqcDhyR5UZJdkyxJ8tT2bOcDwLuTPCrJgiTPbMdEvgUsap9/IfBWYGtjJYuBHwM/SfJE4JUj\n2z4BPDLJ65M8KMniJE8f2f4h4BhgFQaFtsKg0A6tqq6leWf8Xpp37IcDh1fVXVV1F/C7NC+IN9OM\nZ3xsZN+1wLHA3wK3AOvatn28Cjgxye3A22gCa9NxvwscRhNaN9MMZP9Ku/mNwNdoxkpuBt4B7FJV\nt7XHfD/N2dAdwH0+BTWNN9IE1O00oXfWSA2301xWOhz4AfBt4Dkj279AM4j+laoavRwn3U/84iJp\n55Tkn4GPVNX757oWjTeDQtoJJflV4AKaMZbb57oejbfBLj0l+UCSm5J8fQvbk+Q9SdalubHqgKFq\nkbRZklNp7rF4vSGhPgY7o0jym8BPgA9V1ZOn2X4Y8Bqaa7lPB/5XVT19ajtJ0twa7Iyiqj5HM1i3\nJUfQhEhV1SXAnkkeOVQ9kqSZmcuJxfbivjcQTbbrvj+1YZLjaO6i5aEPfejTnvjEJ85KgZI0X1x+\n+eU/qqqp9+b0skPMQFlVq4HVABMTE7V27do5rkiSdixJZvwx6Lm8j+JGmjtoN1nerpMkjZG5DIo1\nwMvaTz89g2a+mftddpIkza3BLj0lOQM4CFjazrP/dpqZPKmqk2mmgz6M5m7YnwIvH6oWSdLMDRYU\nVXXUVrYX8Ort8Vx33303k5OTbNy48X7bFi1axPLly1m40O9kkaSZ2CEGs7dmcnKSxYsXs88++zA6\nUWhVsWHDBiYnJ9l3333nsEJJ2nHNi0kBN27cyJIlS+4TEgBJWLJkybRnGpKkfuZFUAD3C4mtrZck\n9TNvgkKSNAyDQpLUad4ExZYmN3QadUl6YOZFUCxatIgNGzbcLxQ2fepp0aJFc1SZJO345sXHY5cv\nX87k5CTr16+/37ZN91FIkmZmXgTFwoULvU9CkgYyLy49SZKGY1BIkjoZFJKkTgaFJKmTQSFJ6mRQ\nSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQ\nSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoNGhRJDk1ybZJ1Sd48zfZH\nJ7k4yRVJrkpy2JD1SJK23WBBkWQBcBLwXGAlcFSSlVOavRU4u6r2B44E/m6oeiRJMzPkGcWBwLqq\nuq6q7gLOBI6Y0qaAh7WP9wC+N2A9kqQZGDIo9gJuGFmebNeNOgE4OskkcB7wmukOlOS4JGuTrF2/\nfv0QtUqStmCuB7OPAj5YVcuBw4DTktyvpqpaXVUTVTWxbNmyWS9SknZmQwbFjcDeI8vL23WjXgGc\nDVBVXwIWAUsHrEmStI2GDIrLgBVJ9k2yG81g9Zopbb4LHAyQ5Ek0QeG1JUkaI4MFRVXdAxwPnA98\ng+bTTVcnOTHJqrbZG4Bjk3wVOAM4pqpqqJokSdtu1yEPXlXn0QxSj65728jja4BnDVmDJOmBmevB\nbEnSmDMoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJ\noJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJ\noJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0GDYokhya5Nsm6\nJG/eQpsXJbkmydVJPjJkPZKkbbfrUAdOsgA4CfhtYBK4LMmaqrpmpM0K4M+BZ1XVLUkeMVQ9kqSZ\nGfKM4kBgXVVdV1V3AWcCR0xpcyxwUlXdAlBVNw1YjyRpBoYMir2AG0aWJ9t1o/YD9kvyhSSXJDl0\nugMlOS7J2iRr169fP1C5kqTpzPVg9q7ACuAg4CjgfUn2nNqoqlZX1URVTSxbtmyWS5SknVuvoEjy\nsSTPS7ItwXIjsPfI8vJ23ahJYE1V3V1V/wp8iyY4JEljou8L/98BLwG+neSvkjyhxz6XASuS7Jtk\nN+BIYM2UNufSnE2QZCnNpajretYkSZoFvYKiqi6sqt8HDgC+A1yY5ItJXp5k4Rb2uQc4Hjgf+AZw\ndlVdneTEJKvaZucDG5JcA1wMvKmqNjywX0mStD2lqvo1TJYARwMvBb4HnA78OvDLVXXQUAVONTEx\nUWvXrp2tp5OkeSHJ5VU1MZN9e91HkeTjwBOA04DDq+r77aazkviqLUnzWN8b7t5TVRdPt2GmCSVJ\n2jH0HcxeOfqx1SQPT/KqgWqSJI2RvkFxbFXdummhvZP62GFKkiSNk75BsSBJNi208zjtNkxJkqRx\n0neM4tM0A9entMt/1K6TJM1zfYPiz2jC4ZXt8gXA+wepSJI0VnoFRVXdC/x9+yNJ2on0vY9iBfDf\ngZXAok3rq+qxA9UlSRoTfQez/5HmbOIe4DnAh4APD1WUJGl89A2KB1fVRTRTflxfVScAzxuuLEnS\nuOg7mH1nO8X4t5McTzNd+O7DlSVJGhd9zyheBzwEeC3wNJrJAf9gqKIkSeNjq2cU7c11L66qNwI/\nAV4+eFWSpLGx1TOKqvo5zXTikqSdUN8xiiuSrAE+CtyxaWVVfWyQqiRJY6NvUCwCNgC/NbKuAINC\nkua5vndmOy4hSTupvndm/yPNGcR9VNUfbveKJEljpe+lp0+MPF4EvIDme7MlSfNc30tP/3t0OckZ\nwOcHqUiSNFb63nA31QrgEduzEEnSeOo7RnE79x2j+AHNd1RIkua5vpeeFg9diCRpPPW69JTkBUn2\nGFneM8nzhytLkjQu+o5RvL2qbtu0UFW3Am8fpiRJ0jjpGxTTtev70VpJ0g6sb1CsTfLuJI9rf94N\nXD5kYZKk8dA3KF4D3AWcBZwJbARePVRRkqTx0fdTT3cAbx64FknSGOr7qacLkuw5svzwJOcPV5Yk\naVz0vfS0tP2kEwBVdQvemS1JO4W+QXFvkkdvWkiyD9PMJitJmn/6fsT1LcDnk3wWCPAbwHGDVSVJ\nGht9B7M/nWSCJhyuAM4FfjZkYZKk8dB3MPs/AhcBbwDeCJwGnNBjv0OTXJtkXZItfmoqyQuTVBtG\nkqQx0neM4nXArwLXV9VzgP2BW7t2SLIAOAl4LrASOCrJymnaLW6Pf+k21C1JmiV9g2JjVW0ESPKg\nqvom8ISt7HMgsK6qrququ2hu1DtimnZ/CbyD5iY+SdKY6RsUk+19FOcCFyT5J+D6reyzF3DD6DHa\ndf8myQHA3lX1ya4DJTkuydoka9evX9+zZEnS9tB3MPsF7cMTklwM7AF8+oE8cZJdgHcDx/R4/tXA\naoCJiQk/litJs2ibZ4Ctqs/2bHojsPfI8vJ23SaLgScDn0kC8IvAmiSrqmrtttYlSRrGTL8zu4/L\ngBVJ9k2yG3AksGbTxqq6raqWVtU+VbUPcAlgSEjSmBksKKrqHuB44HzgG8DZVXV1khOTrBrqeSVJ\n29egXz5UVecB501Z97YttD1oyFokSTMz5KUnSdI8YFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSS\npE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSS\npE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSS\npE4GhSSpk0EhSepkUEiSOg0aFEkOTXJtknVJ3jzN9j9Nck2Sq5JclOQxQ9YjSdp2gwVFkgXAScBz\ngZXAUUlWTml2BTBRVU8BzgH+x1D1SJJmZsgzigOBdVV1XVXdBZwJHDHaoKourqqftouXAMsHrEeS\nNANDBsVewA0jy5Ptui15BfCp6TYkOS7J2iRr169fvx1LlCRtzVgMZic5GpgA3jnd9qpaXVUTVTWx\nbNmy2S1OknZyuw547BuBvUeWl7fr7iPJIcBbgGdX1Z0D1iNJmoEhzyguA1Yk2TfJbsCRwJrRBkn2\nB04BVlXVTQPWIkmaocGCoqruAY4Hzge+AZxdVVcnOTHJqrbZO4HdgY8muTLJmi0cTpI0R4a89ERV\nnQecN2Xd20YeHzLk80uSHrixGMyWJI0vg0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmd\nDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmd\nDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmd\nDApJUieDQpLUyaCQJHUaNCiSHJrk2iTrkrx5mu0PSnJWu/3SJPsMWY8kadsNFhRJFgAnAc8FVgJH\nJVk5pdkrgFuq6vHA3wDvGKoeSdLMDHlGcSCwrqquq6q7gDOBI6a0OQI4tX18DnBwkgxYkyRpG+06\n4LH3Am4YWZ4Enr6lNlV1T5LbgCXAj0YbJTkOOK5dvDPJ1wepeMezlCl9tROzLzazLzazLzZ7wkx3\nHDIotpuqWg2sBkiytqom5riksWBfbGZfbGZfbGZfbJZk7Uz3HfLS043A3iPLy9t107ZJsiuwB7Bh\nwJokSdtoyKC4DFiRZN8kuwFHAmumtFkD/EH7+PeAf66qGrAmSdI2GuzSUzvmcDxwPrAA+EBVXZ3k\nRGBtVa0B/gE4Lck64GaaMNma1UPVvAOyLzazLzazLzazLzabcV/EN/CSpC7emS1J6mRQSJI6jW1Q\nOP3HZj364k+TXJPkqiQXJXnMXNQ5G7bWFyPtXpikkszbj0b26YskL2r/Nq5O8pHZrnG29PgfeXSS\ni5Nc0f6fHDYXdQ4tyQeS3LSle83SeE/bT1clOaDXgatq7H5oBr//BXgssBvwVWDllDavAk5uHx8J\nnDXXdc9hXzwHeEj7+JU7c1+07RYDnwMuASbmuu45/LtYAVwBPLxdfsRc1z2HfbEaeGX7eCXwnbmu\ne6C++E3gAODrW9h+GPApIMAzgEv7HHdczyic/mOzrfZFVV1cVT9tFy+huWdlPurzdwHwlzTzhm2c\nzeJmWZ++OBY4qapuAaiqm2a5xtnSpy8KeFj7eA/ge7NY36ypqs/RfIJ0S44APlSNS4A9kzxya8cd\n16CYbvqPvbbUpqruATZN/zHf9OmLUa+geccwH221L9pT6b2r6pOzWdgc6PN3sR+wX5IvJLkkyaGz\nVt3s6tMXJwBHJ5kEzgNeMzuljZ1tfT0BdpApPNRPkqOBCeDZc13LXEiyC/Bu4Jg5LmVc7Epz+ekg\nmrPMzyX55aq6dU6rmhtHAR+sqncleSbN/VtPrqp757qwHcG4nlE4/cdmffqCJIcAbwFWVdWds1Tb\nbNtaXywGngx8Jsl3aK7BrpmnA9p9/i4mgTVVdXdV/SvwLZrgmG/69MUrgLMBqupLwCKaCQN3Nr1e\nT6Ya16Bw+o/NttoXSfYHTqEJifl6HRq20hdVdVtVLa2qfapqH5rxmlVVNePJ0MZYn/+Rc2nOJkiy\nlOZS1HWzWeQs6dMX3wUOBkjyJJqgWD+rVY6HNcDL2k8/PQO4raq+v7WdxvLSUw03/ccOp2dfvBPY\nHfhoO57/3apaNWdFD6RnX+wUevbF+cC/T3IN8HPgTVU17866e/bFG4D3JfkTmoHtY+bjG8skZ9C8\nOVjajse8HVgIUFUn04zPHAasA34KvLzXcedhX0mStqNxvfQkSRoTBoUkqZNBIUnqZFBIkjoZFJKk\nTgaFNIuSHJTkE3Ndh7QtDApJUieDQppGkqOTfDnJlUlOSbIgyU+S/E373Q4XJVnWtn1qO+neVUk+\nnuTh7frHJ7kwyVeTfCXJ49rD757knCTfTHL6PJ31WPOIQSFN0U7x8GLgWVX1VJq7mn8feCjNnb6/\nBHyW5q5XgA8Bf1ZVTwG+NrL+dJppvn8F+DVg01QJ+wOvp/lehMcCzxr8l5IegLGcwkOaYwcDTwMu\na9/sPxi4CbgXOKtt82HgY0n2APasqs+260+lmUplMbBXVX0coKo2ArTH+3JVTbbLVwL7AJ8f/teS\nZsagkO4vwKlV9ef3WZn85yntZjr/zejsvj/H/0ONOS89Sfd3EfB7SR4BkOQX2u8h34VmpmKAlwCf\nr6rbgFuS/Ea7/qXAZ6vqdmAyyfPbYzwoyUNm9beQthPfyUhTVNU1Sd4K/N/2y5DuBl4N3AEc2G67\niWYcA5rp7k9ug+A6Ns/I+VLglHYW07uB/zCLv4a03Th7rNRTkp9U1e5zXYc027z0JEnq5BmFJKmT\nZxSSpE4GhSSpk0EhSepkUEiSOhkUkqRO/x9sLpDvMsDn7AAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8XXWd//HXJ0uTNkmTNEnTNl0p\ndKEsBQqCLLJvIosyoAIqLtVRf4MzDIqOODrzc0bH+bmiYFlUFlmEsoiIgICASKHU0pYm3aBL2qZp\n02ZrmzbL5/fH9+Q0TZM0bXJz0+T9fDzu4557zvfe+703N/d9v9/zPd9j7o6IiAhASrIrICIi/YdC\nQUREYgoFERGJKRRERCSmUBARkZhCQUREYgoFkW4ys1+b2f/tZtnVZnZuTx9HpK8pFEREJKZQEBGR\nmEJBBpSo2+YmM1tkZtvN7C4zKzazP5pZnZk9b2b5bcpfambvmFm1mb1kZtPbbDvOzBZE93sIyGz3\nXJeY2cLovq+Z2TEHWefPmdlKM9tqZk+a2ZhovZnZj8ys0sxqzWyxmR0VbbvYzJZGdVtvZv96UG+Y\nSDsKBRmIPgKcB0wBPgT8EfgGUET4zP8TgJlNAR4AvhJtexr4vZkNMbMhwOPAvcAI4HfR4xLd9zjg\nbuDzQAHwS+BJM8s4kIqa2dnAfwNXAaOBNcCD0ebzgTOi15EblamKtt0FfN7dc4CjgBcO5HlFOqNQ\nkIHoZ+6+yd3XA68A89z97+7eADwGHBeVuxr4g7s/5+6NwP8CQ4H3AycD6cCP3b3R3R8B3mzzHLOB\nX7r7PHdvdvffALui+x2Ia4C73X2Bu+8Cvg6cYmYTgUYgB5gGmLuXuvvG6H6NwJFmNtzdt7n7ggN8\nXpEOKRRkINrUZnlnB7ezo+UxhF/mALh7C7AOKIm2rfe9Z4xc02Z5AnBj1HVUbWbVwLjofgeifR3q\nCa2BEnd/AbgV+DlQaWZzzGx4VPQjwMXAGjP7i5mdcoDPK9IhhYIMZhsIX+5A6MMnfLGvBzYCJdG6\nVuPbLK8DvuvueW0uw9z9gR7WIYvQHbUewN1/6u4nAEcSupFuita/6e6XASMJ3VwPH+DzinRIoSCD\n2cPAB83sHDNLB24kdAG9BvwNaAL+yczSzezDwElt7nsH8AUze1+0QzjLzD5oZjkHWIcHgOvNbGa0\nP+K/CN1dq83sxOjx04HtQAPQEu3zuMbMcqNur1qgpQfvg0hMoSCDlrsvA64FfgZsIeyU/pC773b3\n3cCHgU8BWwn7H+a2ue984HOE7p1twMqo7IHW4XngFuBRQutkMvDRaPNwQvhsI3QxVQE/iLZdB6w2\ns1rgC4R9EyI9ZjrJjoiItFJLQUREYgoFERGJKRRERCSmUBARkVhasitwoAoLC33ixInJroaIyCHl\nrbfe2uLuRfsrl7BQMLNxwD1AMeDAHHf/SbsyucB9hIOC0oD/dfdfdfW4EydOZP78+YmptIjIAGVm\na/ZfKrEthSbgRndfEB3Q85aZPefuS9uU+RKw1N0/ZGZFwDIzuz8aIy4iIn0sYfsU3H1j6yRd7l4H\nlBLmlNmrGJATTSWQTThIqClRdRIRka71yY7maMbH44B57TbdCkwnzP+yGLghmpSs/f1nm9l8M5u/\nefPmBNdWRGTwSviOZjPLJhzC/xV3r223+QJgIXA24fD+58zslfbl3H0OMAdg1qxZ+xyC3djYSHl5\nOQ0NDYl4Cf1KZmYmY8eOJT09PdlVEZEBKKGhEE3k9Shwv7vP7aDI9cD3oumJV5rZe4S54984kOcp\nLy8nJyeHiRMnsveklgOLu1NVVUV5eTmTJk1KdnVEZABKWPdRtJ/gLqDU3X/YSbG1wDlR+WJgKvDu\ngT5XQ0MDBQUFAzoQAMyMgoKCQdEiEpHkSGRL4VTCTI6LzWxhtO4bRHPSu/vtwH8CvzazxYABX3P3\nLQfzZAM9EFoNltcpIsmRsFBw91cJX/RdldlAOA9t4jU3QX0F5IyGlNQ+eUoRkUPN4JnmYncdbN8M\nW1ZA065efejq6mp+8YtfHPD9Lr74Yqqrq3u1LiIiPTF4QmFoPoyYDM27YfMyaGg/EOrgdRYKTU1d\nH3Lx9NNPk5eX12v1GLDcobkRmnRMo0iiHXJzH/VI5nAomgpb34Otq0JXUnYx9LCf/uabb2bVqlXM\nnDmT9PR0MjMzyc/Pp6ysjOXLl3P55Zezbt06GhoauOGGG5g9ezawZ8qO+vp6LrroIk477TRee+01\nSkpKeOKJJxg6dGhvvOq+4Q6NO2BnNezctufS0PZ2dWilNe8K1/Hy7nbXHWzzFrBUOPUGOOvfIHVw\nfXRF+sqA+8/6zu/fYemG/bUCPHzptGyBlFJIy6Cr3R9HjhnOv39oRqfbv/e977FkyRIWLlzISy+9\nxAc/+EGWLFkSDxu9++67GTFiBDt37uTEE0/kIx/5CAUFBXs9xooVK3jggQe44447uOqqq3j00Ue5\n9tpru/uyE29XHbz1a6jd2MkX/rbQCutMShpk5kH6UEgdEt7ztAxIzYC0IZCRs++6tMw9ZVMzYMty\nePWHsOY1uPIuyB3bZy9fZLAYcKHQPRa+cJobw6/QxpZw23qnN+2kk07a6ziCn/70pzz22GMArFu3\njhUrVuwTCpMmTWLmzJkAnHDCCaxevbpX6tIrtr4HD3wMNpfCkOzw5T40H4bmQeGUPctD8/dcMtvd\nHpLV4xYZAFMugN/fALefBpffBlMv6vljikhswIVCV7/oO7SrDratDt0f+RMgM7fHdcjKyoqXX3rp\nJZ5//nn+9re/MWzYMM4888wOjzPIyMiIl1NTU9m5c2eP69Er3nsZHv5EeH+uexwmn5Xc+hx9JYw5\nDn73KXjgo3Dyl+Dcb4eWhYj02ODZ0dyZjBwonBq+VLa+G7pHfJ+ZNLqUk5NDXV1dh9tqamrIz89n\n2LBhlJWV8frrr/dGrRPPHd64A+65HLJGwudeSH4gtCqYDJ99Hk76PLz+c7j7ghDsItJjCgUIgVAw\nBYaOCMcybH0XWro/WWtBQQGnnnoqRx11FDfddNNe2y688EKampqYPn06N998MyeffHJv1773Ne2G\np/4Znv5XOOK88AVcMDnZtdpbWgZc/D9w1b1QtQpuPwPeeTzZtRI55Jkf4K/iZJs1a5a3P8lOaWkp\n06dP7/mDu8OOLVCzPuzgHDEp7BjtZ3rt9XZk+xZ46DpY+xqc9i9w9jf7/8F+21bDI5+G9W/BiZ+F\n878L6ZnJrlXv21kd9t2IHAQze8vdZ+2vnFoKbZlBVhEUHA7eHEa77NyW7Fr1nYrFMOcs2LAAPnIX\nnPvv/T8QAPInwvXPwClfhjfvhLvOhS0rk12r3rOrDv74Nfj+xNCCa9lndnmRXqNQ6EhGdjieIW1o\n+BVau/6A9zMccpY+AXedH7rNrv9j2KF7KEkbAhd8Fz72ENSUw5wPwKLfJbtWPVf2NPz8fTDvlzDh\n/TD/bnj8H8O0LSIJoFDoTOoQKDwchhVAfWU42G0g/iO2tMCL/x1GGBXPgNkvQsnxya7VwZt6IXzh\nrzDqaJj7WXjiy7B7x8E9ljvUbggjsN68C/78n1DeR+cHr90YuvEe/FgYEfeZZ+H6p0N33qIH4dFP\n6whvSYgBNyS1V1kK5I2H9GHh1+fmstBXbWmhW6X1Ym2v09qt68ezmu6qh8e/AKW/h5nXwAd/ODD6\n4nNL4JNPwUv/Da/8v/BF/g+/gpGd7IfZVQdVK0OXU9VKqFoR5siqWgWN2/cu+8r/wsTT4fR/gcPO\n6v2/b0sLvPUreP7b4QDLc74F7/8nSI1OqnTGTZCeBX/6OjTuhKvu6Zf7veTQpVDojqzC8I9XVxG6\nV1p2h30O3RmhZO2CozU0UtLaXdqss5TEh8m2NfDgx6FyKVzwX3DyF/t3gB2o1DQ45xaYeCrMnR32\nlVzw3XAU9JYV0Zf/yrBcX9HmjhZ+CBQeEbprCg4PywWHQ8ZwWHAP/O1WuPcKGD0TTvtnmP6h3tn3\nUlkaDsxbNw8mnQGX/LjjUV+nfDF8Hp/6Z/jtVfDRB0KXp0gv0OijnnAPc/K0NEchEQVFvNzZ+qb9\nBIq1C442y6lDKH13HdPHFcLwMQf3ZbT6r/DwdaEOV94Nh5970G/BIaGuAuZ+LnQDtRqaDwVHRF/4\nk/cs50/af2upaRe8/SD89SehW7Hg8DAn0zFXR1OmHKDGhtACefXH4biZC/4Ljv3o/kP67YdCS2/s\nifDxhzUySbrU3dFHCoVeUF1dzW9/+1u++MUvdv9O7uDN/PhHP2L2Zz7FsMwhe8KibXC0vw2Urqlk\n+p+uCvs98saHL7L8iWEIbf7E6PaEMLVEe/N/FY4/yJ8EH3sw7DcZDFqaYdWL4Uu34HDIKtj/fbrz\nmKW/D/MxbXwbcsbAKV+CEz7V/V/u770SWgdbV8GxHwvDaQ+kbkufgEc+A8VHwrWP9c7rkgFJodCH\nVq9ezSWXXMKSJUsO+L6tM6UWFhbuv7A7NO+mtHQp03cthG3vhdFRW6PrXe0mAswu3jswatbB3++D\nw88LE8r1wpQeQvi7rHoBXv0RrH4lzPv0vs+HI647+5LesRWeuyX8PfInhq6igz1ifPmzoeWXPwk+\n8TjkjDrolyIDV3dDQfsUekHbqbPPO+88Ro4cycMPP8yuXbu44oor+M53vsP27du56qqrKC8vp7m5\nmVtuuYVNmzaxYcMGzjrrLAoLC3nxxRe7fiKzaCbRTDj6+r23uYdjKra+F4VFa2CshtWvwqKHAA87\nLc/99qFx/MGhwgwOPydc1r0Jf/0x/OX78NrP4PhPhtZD3rhQ1h0WPwLP3BxmmT3tn+GMr8KQYQf/\n/FPOh2t+B7/9KPzqIvjEk3ueT/qH9QvC/+D4U2DG5cmuTZcGXkvhjzeHg7B606ij4aLvdbq5bUvh\n2Wef5ZFHHuGXv/wl7s6ll17KV7/6VTZv3swzzzzDHXfcAYQ5kXJzcw+spRA5qJZRYwPsrg87zSXx\nKsvCPofFD4fbx1wdLn/9Caz6M5TMgg/9BEYd1XvPue4NuO/KcN6QTzzR/6YmGWxamqHsKfjbL2Dd\n62EAibfA0VfBxT/o831AaikkybPPPsuzzz7LcccdB0B9fT0rVqzg9NNP58Ybb+RrX/sal1xyCaef\nfnrfViw9c2AMNz1UjJwGV9wGZ30dXrs1jFpaeD8MyYGLfgAnfqb3W2vjToJP/T6MjPrVxSEYRk7r\n3ec4EI07w+iuzcvCcO6t78LoY2DGFaHLbKBqqIW/3wvzbofqtZA3AS747zB44M074aXvhXOCXHFb\nGGXWzwy8UOjiF31fcHe+/vWv8/nPf36fbQsWLODpp5/mm9/8Jueccw7f+ta3klBD6VN548PEfR/4\nKiz/Exx2ZjiOIlFGHwufehruuQx+fTFc91hYl0i76sOUMK1f/q3X21YDUU+EpYYzHb4zNxyDMeZ4\nOOrDISAGysmStq0OR54vuDecE378KWHgwLQP7vkB8IGvhm7GubPhN5eGrsWzb+lXP9gGXigkQdup\nsy+44AJuueUWrrnmGrKzs1m/fj3p6ek0NTUxYsQIrr32WvLy8rjzzjv3uu+BdB/JISirEI67pm+e\na+S0cPTzPZfBrz8E1z4SWhE91VAT/fIv2/PlX1kGNWv3lElJD6O7Rh8busuKpkLRtNCVlZYRjo95\n57Fwefab4TL2pBAQR14Ow0f3vJ59yR3Wvh6mcC/7Q+gimnFFOO6ns5kBSk6Az78SBhr87dYwKu7D\nc3q3K7EHBt4+hST5+Mc/zqJFi7jooosYO3Zs/KWfnZ3Nfffdx8qVK7nppptISUkhPT2d2267jVmz\nZvGzn/2MW2+9lTFjxux/R3OkP7xeOQRUrw3BULcJPv4QTNpPl6V7GBW17b3Q1RNfots7tuwpm5oR\nzrrX+qXfej1i0p6jr/enalUUEI/DpsWAhV/XR30YjrwMskce9EtPuObGUO/Xfw4b/h5GnM26Hk78\n3IG1BFc8B098KQwSOfubYVLHBA0C0ZDUAWywvV7pgbqKEAzbVsPV94UDFesqOvnifw921bS5s8Hw\nkvBFP+KwcF04NQRA/sTe/fLavDwKiLmhFWIpMOHUEBDTL+0/AyR2bA3nKn/jDqjbEFpFJ/9jOMak\no+OCumN7FTx1QzjmZcJpYV9D3vherTYoFAa0wfZ6pYe2V8G9l4cpTVKHQGObCQItNRzomN/6xX/Y\nnhDIm5Ccvu7KUlgyNwRE1cpQx0lnhG6Zomlh1E5mXjjOprfr19ICO7fC9s3hUl8ZzjGyfXNoeZU9\nFd6/SR8I+wMOPw9SemFeUXd4+wF4+qthiPPFPwjdb7049YxCYQAbbK9XesHO6jDqxVL2/uWfO677\n3T19zR02LdkTEB2dcjUtMwTE0Cgk4uXoduvy0Lwwd9Xu7dEXfpsv+7Zf/Du2hGGj7VlqaK0cfl5o\nGSSq/3/banjsC7D2b2EfyyU/gmEjeuWhB10oTJs2DRtIE7p1wt0pKytTKMjg4h5aELUbwkF/DdUh\n6OLrmr3XNdSEoaF08f02JCd80WePDCfXyioM5yOPl4v2bMvM650WQXe0NMNrP4UXvhum7r/8F2HE\nUg8NquMUMjMzqaqqoqCgYEAHg7tTVVVFZmb/Gb4m0ifMwvxOxUd2/z4tzWHql4aaPcGRkR198Rf2\n3ynHU1LDke6Tzw5DV+/7MJw0G879Ts+OfO+mAdFSaGxspLy8nIaGhiTVqu9kZmYyduxY0tP7aZNf\nRHpPYwP8+T/CKKfCKWHo6pjjDuqhkt5SMLNxwD1AMaENN8fdf9JBuTOBHwPpwBZ3/8CBPld6ejqT\nJk3qWYVFRPqb9Ey48L/C/FaPfzGcnvUgQ6G7Etl91ATc6O4LzCwHeMvMnnP3pa0FzCwP+AVwobuv\nNbN+PDBZRCRJDjsT/vGvMCTxJ1NK2J4Td9/o7gui5TqgFGh/VMfHgbnuvjYqV5mo+oiIHNKG5vfJ\nSLE+2Z1uZhOB44B57TZNAfLN7CUze8vMPtHJ/Web2Xwzm7958+bEVlZEZBBLeCiYWTbwKPAVd293\nFhjSgBOADwIXALeY2ZT2j+Huc9x9lrvPKioqSnSVRUQGrYQOSTWzdEIg3O/uczsoUg5Uuft2YLuZ\nvQwcCyxPZL1ERKRjCWspWDhg4C6g1N1/2EmxJ4DTzCzNzIYB7yPsexARkSRIZEvhVOA6YLGZLYzW\nfQMYD+Dut7t7qZk9AywCWoA73f3AT3QsIiK9ImGh4O6vAvs9vNjdfwD8IFH1EBGR7uujyTxERORQ\noFAQEZGYQkFERGIKBRERiSkUREQkplAQEZGYQkFERGIKBRERiSkUREQkplAQEZGYQkFERGIKBRER\niSkUREQkplAQEZGYQkFERGIKBRERiSkUREQkplAQEZGYQkFERGIKBRERiSkUREQkplAQEZGYQkFE\nRGIKBRERiSkUREQkplAQEZGYQkFERGIKBRERiSkUREQklrBQMLNxZvaimS01s3fM7IYuyp5oZk1m\ndmWi6iMiIvuXlsDHbgJudPcFZpYDvGVmz7n70raFzCwV+D7wbALrIiIi3ZCwloK7b3T3BdFyHVAK\nlHRQ9P8AjwKViaqLiIh0T5/sUzCzicBxwLx260uAK4Db9nP/2WY238zmb968OVHVFBEZ9BIeCmaW\nTWgJfMXda9tt/jHwNXdv6eox3H2Ou89y91lFRUWJqqqIyKCXyH0KmFk6IRDud/e5HRSZBTxoZgCF\nwMVm1uTujyeyXiIi0rGEhYKFb/q7gFJ3/2FHZdx9UpvyvwaeUiCIiCRPIlsKpwLXAYvNbGG07hvA\neAB3vz2Bzy0iIgchYaHg7q8CdgDlP5WouoiISPfoiGYREYkpFEREJKZQEBGRmEJBRERiCgUREYkp\nFEREJKZQEBGRmEJBRERiCgUREYkpFEREJKZQEBGRmEJBRERiCgUREYkpFEREJKZQEBGRmEJBRERi\nCgUREYl1KxTM7AYzG27BXWa2wMzOT3TlRESkb3W3pfBpd68FzgfyCede/l7CaiUiIknR3VBoPdfy\nxcC97v4OB3D+ZREROTR0NxTeMrNnCaHwJzPLAVoSVy0REUmGtG6W+wwwE3jX3XeY2Qjg+sRVS0RE\nkqG7LYVTgGXuXm1m1wLfBGoSVy0REUmG7obCbcAOMzsWuBFYBdyTsFqJiEhSdDcUmtzdgcuAW939\n50BO4qolIiLJ0N19CnVm9nXCUNTTzSwFSE9ctUREJBm621K4GthFOF6hAhgL/CBhtRIRkaToVihE\nQXA/kGtmlwAN7q59CiIiA0x3p7m4CngD+AfgKmCemV2ZyIqJiEjf6+4+hX8DTnT3SgAzKwKeBx7p\n7A5mNo4wQqkYcGCOu/+kXZlrgK8Rjo6uA/7R3d8+0BchIiK9o7uhkNIaCJEq9t/KaAJudPcF0RHQ\nb5nZc+6+tE2Z94APuPs2M7sImAO8r7uVFxGR3tXdUHjGzP4EPBDdvhp4uqs7uPtGYGO0XGdmpUAJ\nsLRNmdfa3OV1wg5sERFJkm6FgrvfZGYfAU6NVs1x98e6+yRmNhE4DpjXRbHPAH/s5P6zgdkA48eP\n7+7TiojIAbJwTFoCn8AsG/gL8F13n9tJmbOAXwCnuXtVV483a9Ysnz9/fu9XVERkADOzt9x91v7K\nddlSMLM6wk7ifTYB7u7D93P/dOBR4P4uAuEY4E7gov0FgoiIJFaXoeDuBz2VhZkZcBdQ6u4/7KTM\neGAucJ27Lz/Y5xIRkd7R3R3NB+NUwrQYi81sYbTuG8B4AHe/HfgWUAD8ImQITd1p3oiISGIkLBTc\n/VX2c3Y2d/8s8NlE1UFERA5Md+c+EhGRQUChICIiMYWCiIjEFAoiIhJTKIiISEyhICIiMYWCiIjE\nFAoiIhJTKIiISEyhICIiMYWCiIjEFAoiIhJTKIiISEyhICIiMYWCiIjEFAoiIhJTKIiISEyhICIi\nMYWCiIjEFAoiIhJTKIiISEyhICIiMYWCiIjEFAoiIhJTKIiISEyhICIiMYWCiIjEFAoiIhJTKIiI\nSCxhoWBm48zsRTNbambvmNkNHZQxM/upma00s0Vmdnyi6iMiIvuXlsDHbgJudPcFZpYDvGVmz7n7\n0jZlLgKOiC7vA26LrkVEJAkSFgruvhHYGC3XmVkpUAK0DYXLgHvc3YHXzSzPzEZH9+1VKzbV8eCb\n65g+ejjTRuVwRHE2GWmpvf00IiKHtES2FGJmNhE4DpjXblMJsK7N7fJo3V6hYGazgdkA48ePP6g6\nrKys5/55a2hobAEgLcWYXJTNtNE5cVAcOXo4RTkZmNlBPYeIyKEu4aFgZtnAo8BX3L32YB7D3ecA\ncwBmzZrlB/MYFx09mvNnjGJ11XZKN9ZStrGO0o21vPneVp5YuCEuV5A1JATFqOEhLEbncPhItSpE\nZHBIaCiYWTohEO5397kdFFkPjGtze2y0LiFSo9bB5KJsLjlmz/qaHY2UVtTuCYuKWu59fQ27mvZu\nVUwfncPUUaFVMWVUDmNyM9WqEJEBJWGhYOHb8i6g1N1/2EmxJ4Evm9mDhB3MNYnYn7A/ucPSOfmw\nAk4+rCBe19TcwuqqHZRujMKioo55723l8TatipzMNKYW5zB1VE4IiuIcpo0aTu6w9L5+CSIivcLC\nPt4EPLDZacArwGKgJVr9DWA8gLvfHgXHrcCFwA7genef39Xjzpo1y+fP77JIQtXsaGR5ZR1lFXUs\nq6hlWUVYrmtoisuMGp65V1BMHRW6oDLT1QUlIslhZm+5+6z9lktUKCRKskOhI+7OxpoGllXUsWxT\nXRwUqyrr2d0c8jA1xZhUmMXRJbkcMzaXY8bmMWPMcAWFiPSJ7oZCn4w+GujMjDF5QxmTN5Szpo2M\n1zc2t7B6y3bKKupYvins2H515RYe+3vYbZKWYkwpzuHYcbkcOzaPY8bmMaU4m7RUHWguIsmhUEig\n9NQUjijO4YjinHidu1NR28Db62pYVF7NovIa/rBoIw+8EUbmZqanMGNMaE2EoMhlYkEWKSnaoS0i\niafuo37A3VldtYNF5dVxWCzZUBMfU5GTmRaHxLHj8jhuXB4jh2cmudYicihR99EhxCzsb5hUmMVl\nM0uAMPppRWV9CIryEBRzXn6XppYQ4mNyM5k5Po9jx+Yxc1weR4/NZdgQ/TlFpGf0LdJPpaWmMH10\nOIDu6hPDuobGZt7ZUMPCdTUsXFfNwnXbeHpxBRB2ZE8pzmHmuDxmjstl5rh8Dh+ZTaq6nUTkACgU\nDiGZ6amcMGEEJ0wYEa/bUr+LReXVLFxbzd/XVfOHRRt44I21AGRnpHF0SS4zx+dFYZHHSE3jISJd\nUCgc4gqzMzh7WjFnTysGoKXFea9qO2+vq45aE9Xc+cq7NDaHbqfWaTymRUdmTxs1nCOKdQyFiAQK\nhQEmpc1UHh8+fiwQup2Wbqxl0bpqyirqKK2o22tywBSDSYVZTBs9nOlRUEwbnUNJ3lC1KkQGGYXC\nIJCZnsrx4/M5fnx+vK65xVm7dQdlG2spraijbGMti6Phsa1yMtLCkdltWhZTRuUwPFPTeIgMVBqS\nKnup39XE8k11lG2so6xizwSBbafxGJ2byRHFOUwtzo6uw/kpNPpJpP/SkFQ5KNkZafu0KtydDTUN\nlG2sZfmmepZvCkdo/+bdKnY3tcTlxo0YypSRoTUxpTibKcU5TC7S/gqRQ4lCQfbLzCjJG0pJ3lDO\nmV4cr2/tglpWUceKTWHepxWb6nl5xeZ4x3aKwYSCLKYUZzN11HBOnVzA8RPySddUHiL9krqPpNe1\nzvm0bFNdaFlU1LG8so7VW7bT4uEI7TOmFHHW1JGcObWIwuyMZFdZZMBT95EkTUdzPgHUNTTy6oot\nvLiskheXbY53ah87Npczp47k7GkjObokV/M8iSSRWgqSFC0tztKNtbxYVsmLyyr5+7pq3KEwewgf\nmDKSs6YVcfoRReQO1Ugnkd6g8ynIIWXr9t28vHwzL5RV8pflm6nZ2UhqinHChHzOiloRU4qzddyE\nyEFSKMghq6m5hbfLq3mhrJIXyzazdGMtEIbCTi7KpiQ6d8WYvExK8sMO8FG5mWSkaZSTSGcUCjJg\nVNQ08NKySv66qoq1W3ewoXrL7IGIAAAOj0lEQVQnm+t27VOuKCcjHiU1Ji+TMfFyuM4blq6Whgxa\nCgUZ0HY1NVNR08D6bTtZX72TDdUNbKjeyYaa1ts742k8Wg0bksq4/GGMLxjGhBHDmFAwjPEFWUwY\nMYyS/KEaJisDmkYfyYCWkZbKhIIsJhRkdbjd3dm6fTcbqhvikCjftpO1W3ewpmo7r6zYvFdopKYY\nY/IymViQxfjWwBiRFV0PIytD/yoyOOiTLgOSmVGQnUFBdgZHj83dZ7u7U1m3izVVISRCWOxgzdYd\n/GHxRqp3NO5VvjA7g4kFwzgiOlJ7ajRx4IisIX31kkT6hEJBBiUzo3h4JsXDMzlp0oh9ttfsbGRt\n1Q7WbN3OmqodrK3aweqq7TyzpCI+nzaEsJg2KoTE1CgsNA+UHMr0yRXpQO7QdI4em7tPK8Pd2Vy/\ni2UVdXsum/aeitwMxo8YFodEaFXkMLEgizTtt5B+TqEgcgDMjJE5mYzMyeT0I4ri9W3ngQpBUcuy\nijqeL91EdFpthqSmMGVUNkeNyWXGmOHMKMll+qjhDB2iobTSf2j0kUgCNTQ2s7IyzCxbVlHHOxtq\neGdDbbzPIsVgclF2CIkxucwoGc6M0bnkDtOR3NK7NPpIpB/ITE/lqJJcjirZ0w3VOhX5kvUhIJZu\nqOH1d7fy+MINcZlxI4YyY3RoURxVEq5HDs9MxkuQQUahINLH2k5FfsGMUfH6qvpdvLOhliUbWsOi\nlmfeqYi3F+VkcOzYXGaOy+PYcXkcMzZPc0NJr1MoiPQTBdkZnDGliDOm7NlXUdfQSOnG0O20eH0N\nb6+r5vnSynj75KIsZo7LZ+b4PGaOzWPa6BwdhCc9olAQ6cdyMtM5adKIvYbN1uxsZHF5DQvXbWPh\numr+srySRxeUA5CRlsJRJXtaE8eNy2Ns/lBN7yHdlrAdzWZ2N3AJUOnuR3WwPRe4DxhPCKf/dfdf\n7e9xtaNZZG/uTvm2nbxdXs3CtdUsXFfN4vU17IpOlVqQNYSZUXdTTmYaKQYpKYYRurLMIMXC7ZTo\ntpmRYsTbANJSUphYOIwpxWqNHIqSPveRmZ0B1AP3dBIK3wBy3f1rZlYELANGufvurh5XoSCyf43N\nLSyrqGPhuur4smpzPb3x7z4kLYXpo3I4qiSXo6Od6FOKcxiSpqDoz5I++sjdXzaziV0VAXIstGuz\nga1AU6LqIzKYpKemxKOerj15AgA7dzezq6mZFg+tixYHx3EHd2hxp8X33HZ8r7KNzS2sqKxnyfoa\nFpfX8OTbG7h/3logHIMxbfSeoDhaQXHISuhxClEoPNVJSyEHeBKYBuQAV7v7Hzp5nNnAbIDx48ef\nsGbNmkRVWUS6qSU6YG/x+poQFNF1bUP4bZeeakwdlRO3Jo4pyWP66Bwd1Z0kSe8+iioxkc5D4Urg\nVOBfgMnAc8Cx7l7b1WOq+0ik/3LfExRxWJTvCYr8YemcNW0k500v5vQpRWRr9tk+k/Tuo264Hvie\nh1RaaWbvEVoNbySxTiLSA2YWT2l+yTFjgBAU67buZGF5NS+VVfJCWSVzF6xnSGoKp0wu4Nwjizl3\n+khG5w5Ncu0FkhsKa4FzgFfMrBiYCrybxPqISAKYGeMLwsmNLj12DE3NLcxfs43nl27iudJN3PL4\nEm55HI4uyeXc6cWce+RIjhw9XMNokySRo48eAM4ECoFNwL8D6QDufruZjQF+DYwGjNBquG9/j6vu\nI5GBw91Ztbme55ZW8nzpJhas3YY7jMnNjFoQxbzvsBE6/3Yv6Bf7FBJBoSAycG2p38ULZZU8v3QT\nr6zYws7GZrIz0vjAlCLOibqYUlMsvqS1WU5NMVItWp+6Z7n1kp6aQmb64A0XhYKIHNIaGpt5bdWW\nuBWxuW5Xjx9zZE4GR5fkMqPN0Nni4RmDoqtKoSAiA0ZLi1NWUUfNzkZa3GlqcZpbWmhuIb5uammh\nucXjS1NLOO6iqTnc3t3cwqrKehavr2HV5vr4PBeF2RkcVTI8hMWYcGKlMbmZAy4oDoXRRyIi3ZKS\nYhw5ZnivPd6O3U2UbqxlcXkNi9fX8s6GGl5ZsYXmKClGZA1hxpjhcWviqJLcQTOHlEJBRAadYUPS\nOGHCCE6YsGeiwZ27mymtqOWd6BiLxetrmfPyuzRFQZE3LD0+xeqU1uuROQPuhEgKBRERYOiQVI4f\nn8/x4/PjdQ2NzSyrqGPx+hre2VBDWUUdcxesp37Xnhl5iodnMKU4CoriHKaMyuGIkdlkHaIH5h2a\ntRYR6QOZ6akcG01D3qr1zHnLK+pYvqmOZZvC9f3z1tDQ2BKXG5s/lKnFORxRnMPUUdkcMTKHMXlD\nyRuaTkpK/+2GUiiIiByAtmfOO2vayHh9c4uzbusOlm9qDYt6Vmyq4+UVm2ls3jOgJ8Ugf9gQRmQN\nIT9rCAVZYbkguh2WMxgRLY/IGtKnEwsqFEREekFqijGxMIuJhVmc3+Y0q43NLazesp0VlfVsqm1g\n6/bdVG3fzbboekVlPdu272bbjt3xiKj2cjLSGJE9hOtOnsBnTz8soa9DoSAikkDpqSkcEXUjdaW5\nxanZ2cjW7bvYuj1ctw2Prdt3U5STkfD6KhRERPqB1BSLu4uSSRObi4hITKEgIiIxhYKIiMQUCiIi\nElMoiIhITKEgIiIxhYKIiMQUCiIiEjvkTrJjZpuBNQd590JgSy9Wp7f19/pB/6+j6tczql/P9Of6\nTXD3ov0VOuRCoSfMbH53zjyULP29ftD/66j69Yzq1zP9vX7doe4jERGJKRRERCQ22EJhTrIrsB/9\nvX7Q/+uo+vWM6tcz/b1++zWo9imIiEjXBltLQUREuqBQEBGR2IAMBTO70MyWmdlKM7u5g+0ZZvZQ\ntH2emU3sw7qNM7MXzWypmb1jZjd0UOZMM6sxs4XR5Vt9Vb/o+Veb2eLoued3sN3M7KfR+7fIzI7v\nw7pNbfO+LDSzWjP7Srsyff7+mdndZlZpZkvarBthZs+Z2YroOr+T+34yKrPCzD7Zh/X7gZmVRX/D\nx8wsr5P7dvl5SGD9vm1m69v8HS/u5L5d/r8nsH4PtanbajNb2Ml9E/7+9Sp3H1AXIBVYBRwGDAHe\nBo5sV+aLwO3R8keBh/qwfqOB46PlHGB5B/U7E3gqie/haqCwi+0XA38EDDgZmJfEv3UF4aCcpL5/\nwBnA8cCSNuv+B7g5Wr4Z+H4H9xsBvBtd50fL+X1Uv/OBtGj5+x3VrzufhwTW79vAv3bjM9Dl/3ui\n6tdu+/8DvpWs9683LwOxpXASsNLd33X33cCDwGXtylwG/CZafgQ4x8ysLyrn7hvdfUG0XAeUAiV9\n8dy96DLgHg9eB/LMbHQS6nEOsMrdD/YI917j7i8DW9utbvs5+w1weQd3vQB4zt23uvs24Dngwr6o\nn7s/6+5N0c3XgbG9/bzd1cn71x3d+X/vsa7qF313XAU80NvPmwwDMRRKgHVtbpez75duXCb6p6gB\nCvqkdm1E3VbHAfM62HyKmb1tZn80sxl9WjFw4Fkze8vMZnewvTvvcV/4KJ3/Iybz/WtV7O4bo+UK\noLiDMv3lvfw0ofXXkf19HhLpy1H31t2ddL/1h/fvdGCTu6/oZHsy378DNhBD4ZBgZtnAo8BX3L22\n3eYFhC6RY4GfAY/3cfVOc/fjgYuAL5nZGX38/PtlZkOAS4HfdbA52e/fPjz0I/TL8d9m9m9AE3B/\nJ0WS9Xm4DZgMzAQ2Erpo+qOP0XUrod//P7U1EENhPTCuze2x0boOy5hZGpALVPVJ7cJzphMC4X53\nn9t+u7vXunt9tPw0kG5mhX1VP3dfH11XAo8Rmuhtdec9TrSLgAXuvqn9hmS/f21sau1Wi64rOyiT\n1PfSzD4FXAJcEwXXPrrxeUgId9/k7s3u3gLc0cnzJvv9SwM+DDzUWZlkvX8HayCGwpvAEWY2Kfo1\n+VHgyXZlngRaR3lcCbzQ2T9Eb4v6H+8CSt39h52UGdW6j8PMTiL8nfoktMwsy8xyWpcJOyOXtCv2\nJPCJaBTSyUBNm26SvtLpr7Nkvn/ttP2cfRJ4ooMyfwLON7P8qHvk/GhdwpnZhcBXgUvdfUcnZbrz\neUhU/drup7qik+ftzv97Ip0LlLl7eUcbk/n+HbRk7+lOxIUwOmY5YVTCv0Xr/oPw4QfIJHQ7rATe\nAA7rw7qdRuhGWAQsjC4XA18AvhCV+TLwDmEkxevA+/uwfodFz/t2VIfW969t/Qz4efT+LgZm9fHf\nN4vwJZ/bZl1S3z9CQG0EGgn92p8h7Kf6M7ACeB4YEZWdBdzZ5r6fjj6LK4Hr+7B+Kwn98a2fw9YR\neWOAp7v6PPRR/e6NPl+LCF/0o9vXL7q9z/97X9QvWv/r1s9dm7J9/v715kXTXIiISGwgdh+JiMhB\nUiiIiEhMoSAiIjGFgoiIxBQKIiISUyiI9KFoBtenkl0Pkc4oFEREJKZQEOmAmV1rZm9Ec+D/0sxS\nzazezH5k4TwYfzazoqjsTDN7vc15CfKj9Yeb2fPRxHwLzGxy9PDZZvZIdC6D+/tqhl6R7lAoiLRj\nZtOBq4FT3X0m0AxcQziSer67zwD+Avx7dJd7gK+5+zGEI3Bb198P/NzDxHzvJxwRC2Fm3K8ARxKO\neD014S9KpJvSkl0BkX7oHOAE4M3oR/xQwmR2LeyZ+Ow+YK6Z5QJ57v6XaP1vgN9F892UuPtjAO7e\nABA93hsezZUTna1rIvBq4l+WyP4pFET2ZcBv3P3re600u6VduYOdI2ZXm+Vm9H8o/Yi6j0T29Wfg\nSjMbCfG5licQ/l+ujMp8HHjV3WuAbWZ2erT+OuAvHs6qV25ml0ePkWFmw/r0VYgcBP1CEWnH3Zea\n2TcJZ8tKIcyM+SVgO3BStK2SsN8BwrTYt0df+u8C10frrwN+aWb/ET3GP/ThyxA5KJolVaSbzKze\n3bOTXQ+RRFL3kYiIxNRSEBGRmFoKIiISUyiIiEhMoSAiIjGFgoiIxBQKIiIS+/9AfYPw3pga4wAA\nAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"metadata":{"id":"BQ635w_ONfU5","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"uEsK4tJSNYc6","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":655},"outputId":"f5606fee-63b6-4ebd-beea-3d09f53b61e9","executionInfo":{"status":"ok","timestamp":1556312299707,"user_tz":420,"elapsed":2026,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}}},"cell_type":"code","source":["input_data = next(iter(d))\n","print('input data is ', input_data)"],"execution_count":14,"outputs":[{"output_type":"stream","text":["input data is  (<tf.Tensor: id=67682, shape=(128, 2, 768), dtype=float32, numpy=\n","array([[[-0.04685886, -0.10289637, -0.4312089 , ...,  0.6716723 ,\n","          0.03079298,  0.00706173],\n","        [ 0.17568265,  0.17470282, -0.07537014, ...,  0.3922773 ,\n","          0.218182  , -0.27046928]],\n","\n","       [[-0.05585075,  0.24309337, -0.15819737, ...,  0.3807708 ,\n","          0.11023071,  0.05866839],\n","        [ 0.15998061,  0.41866085,  0.1743543 , ...,  0.41534692,\n","          0.38362348, -0.04816403]],\n","\n","       [[-0.12419132,  0.13910837, -0.11075334, ...,  0.38130343,\n","         -0.07294474, -0.24760696],\n","        [ 0.05594226,  0.17657183, -0.32214007, ...,  0.2297752 ,\n","         -0.02246851,  0.10338924]],\n","\n","       ...,\n","\n","       [[-0.08374447,  0.4017603 , -0.18001199, ...,  0.22693412,\n","          0.23821752,  0.02205274],\n","        [-0.17068203,  0.1651842 , -0.3762887 , ...,  0.3074351 ,\n","          0.186731  , -0.11767841]],\n","\n","       [[ 0.18684188,  0.34592396, -0.03621193, ...,  0.25407133,\n","          0.22771598, -0.04332799],\n","        [-0.08740605,  0.05408696, -0.27524713, ...,  0.41144112,\n","          0.3068106 , -0.19383387]],\n","\n","       [[-0.18405366,  0.2053605 , -0.09115985, ...,  0.24770322,\n","         -0.12814572,  0.04166843],\n","        [-0.00358994,  0.17153965, -0.10701425, ...,  0.5721032 ,\n","          0.22211893, -0.28452796]]], dtype=float32)>, <tf.Tensor: id=67683, shape=(128,), dtype=int64, numpy=\n","array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])>)\n"],"name":"stdout"}]},{"metadata":{"id":"WlcZqjhI3Xuv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1075},"outputId":"cbfb9fa2-cebe-42da-d8b0-67a0e244d6b3","executionInfo":{"status":"ok","timestamp":1556311921584,"user_tz":420,"elapsed":3256,"user":{"displayName":"Santosh Gupta","photoUrl":"","userId":"14163791406641115409"}}},"cell_type":"code","source":["K.set_learning_phase(0)\n","\n","input_data = next(iter(d))[0]\n","print('input data is ', input_data)\n","\n","\n","q_embedding, a_embedding = tf.unstack(medical_qa_model(input_data), axis=1)\n","\n","q_embedding = q_embedding / tf.norm(q_embedding, axis=-1, keepdims=True)\n","a_embedding = a_embedding / tf.norm(a_embedding, axis=-1, keepdims=True)\n","\n","print('q_embedding', q_embedding)\n","print('a_embedding', a_embedding)\n","\n","batch_score = tf.reduce_sum(q_embedding*a_embedding, axis=-1)\n","baseline_score = tf.reduce_mean(tf.matmul(q_embedding,tf.transpose(a_embedding)))\n","\n","print('Training Batch Cos similarity')\n","print(tf.reduce_mean(batch_score))\n","print('Baseline: {0}'.format(baseline_score))"],"execution_count":11,"outputs":[{"output_type":"stream","text":["input data is  tf.Tensor(\n","[[[-0.01894781  0.1705342  -0.23431076 ...  0.2526984   0.03581299\n","   -0.04693769]\n","  [-0.00811298  0.0787234   0.12227986 ...  0.11041731  0.0836745\n","   -0.01399889]]\n","\n"," [[ 0.25619757  0.20239824 -0.17506284 ...  0.29687408  0.02388489\n","   -0.4853307 ]\n","  [ 0.06884078  0.02883266  0.02382688 ...  0.17812629  0.17053054\n","   -0.235822  ]]\n","\n"," [[-0.13511476  0.30704442 -0.01675423 ...  0.48951083  0.04952382\n","   -0.08286653]\n","  [-0.15257362  0.20668145  0.08815259 ...  0.4345324   0.09320357\n","    0.04498511]]\n","\n"," ...\n","\n"," [[ 0.27033815 -0.08848304 -0.34093153 ...  0.23688368  0.232592\n","   -0.1686028 ]\n","  [-0.00228444  0.20956993 -0.39943475 ...  0.49659503  0.26085627\n","   -0.19215202]]\n","\n"," [[ 0.00955392  0.28275514  0.04052498 ...  0.14644516  0.18197495\n","   -0.16054386]\n","  [-0.04791677  0.18572612  0.00437022 ...  0.13283327  0.06079603\n","    0.17652841]]\n","\n"," [[-0.22888634  0.09931215 -0.06191296 ...  0.36640286  0.03239492\n","   -0.31243005]\n","  [-0.1376928   0.29751766 -0.5105773  ...  0.02450288  0.31125784\n","    0.06562237]]], shape=(128, 2, 768), dtype=float32)\n","q_embedding tf.Tensor(\n","[[-0.00081801  0.00736223 -0.01011556 ...  0.01090939  0.0015461\n","  -0.00202637]\n"," [ 0.01098058  0.00867475  0.03886056 ...  0.01272397  0.0010237\n","  -0.02080119]\n"," [-0.00535635  0.01217216  0.00434346 ...  0.06249013  0.01093919\n","  -0.00328508]\n"," ...\n"," [ 0.01203414 -0.00393884 -0.01517662 ...  0.01054491  0.01035387\n","  -0.00750538]\n"," [ 0.00039642  0.01173233  0.06495044 ...  0.00607644  0.00755067\n","  -0.00666143]\n"," [-0.00995191  0.00431806 -0.00269196 ...  0.02807096  0.00349654\n","  -0.01358436]], shape=(128, 768), dtype=float32)\n","a_embedding tf.Tensor(\n","[[-2.9531264e-04  6.7012995e-02  4.4509913e-03 ...  3.4142613e-02\n","   2.8052375e-02 -5.0956011e-04]\n"," [ 3.0872491e-03  5.6023762e-02  1.0685456e-03 ...  2.5012953e-02\n","   7.6476512e-03 -1.0575728e-02]\n"," [-6.3485554e-03  8.5999705e-03  3.6680105e-03 ...  2.2372905e-02\n","   3.8781804e-03  1.8718209e-03]\n"," ...\n"," [-9.9828205e-05  9.1580553e-03 -1.7455012e-02 ...  2.9184271e-02\n","   1.1399232e-02 -8.3969058e-03]\n"," [-2.0442598e-03  1.9250082e-02  1.8644528e-04 ...  1.2289301e-02\n","   2.5937241e-03  7.5311824e-03]\n"," [-5.8287675e-03  1.2594423e-02 -2.1613596e-02 ...  1.0372481e-03\n","   1.3176068e-02  2.7779052e-03]], shape=(128, 768), dtype=float32)\n","Training Batch Cos similarity\n","tf.Tensor(0.7608291, shape=(), dtype=float32)\n","Baseline: 0.7333167791366577\n"],"name":"stdout"}]},{"metadata":{"id":"T3-Y1g6w3jbT","colab_type":"code","colab":{}},"cell_type":"code","source":["eval_d = create_dataset_for_ffn(data_dir='/content/gdrive/My Drive/mqa-biobert', mode='eval', batch_size=64)\n","q_embedding, a_embedding = medical_qa_model(next(iter(eval_d)))\n","\n","q_embedding = q_embedding / tf.norm(q_embedding, axis=-1, keepdims=True)\n","a_embedding = a_embedding / tf.norm(a_embedding, axis=-1, keepdims=True)\n","\n","batch_score = tf.reduce_sum(q_embedding*a_embedding, axis=-1)\n","baseline_score = tf.reduce_mean(tf.matmul(q_embedding,tf.transpose(a_embedding)))\n","\n","print('Eval Batch Cos similarity')\n","print(tf.reduce_mean(batch_score))\n","print('Baseline: {0}'.format(baseline_score))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"0yzF9EWI3mJK","colab_type":"code","colab":{}},"cell_type":"code","source":["d = create_dataset_for_ffn(\n","  data_path, batch_size=batch_size, shuffle_buffer=shuffle_buffer, prefetch=prefetch)\n","eval_d = create_dataset_for_ffn(\n","  data_path, batch_size=batch_size, mode='eval')\n","\n","# save arrays\n","from itertools import chain\n","\n","q_embedding_list = []\n","a_embedding_list = []\n","\n","for feature_dict in chain(iter(d), iter(eval_d)):\n","  q_embedding, a_embedding = tf.unstack(medical_qa_model(feature_dict[0]), axis=1)\n","\n","  q_embedding_list.append(q_embedding / tf.norm(q_embedding, axis=-1, keepdims=True))\n","  a_embedding_list.append(a_embedding / tf.norm(a_embedding, axis=-1, keepdims=True))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"lqDLuzfG3n-b","colab_type":"code","colab":{}},"cell_type":"code","source":["result_path = '/content/gdrive/My Drive/mqa_ffn_results/'\n","os.makedirs(result_path, exist_ok=True)\n","np.save(os.path.join(result_path, 'q_embedding.npz'), np.concatenate(q_embedding_list, axis=0))\n","  \n","\n","np.save(os.path.join(result_path, 'a_embedding.npz'), np.concatenate(a_embedding_list, axis=0) )\n","  "],"execution_count":0,"outputs":[]},{"metadata":{"id":"OxVXFg8L29wa","colab_type":"code","colab":{}},"cell_type":"code","source":["# medical_qa_model.save_weights(model_path)"],"execution_count":0,"outputs":[]}]}